Version: 1.2 (Jupytext, time measurements, logger, param notebook execution, fixes)
Sample data sets for testing anomaly detection pipelines and algorithms.
General settings:
abs_values - If values are bigger or equal to zero.add_zeros - If to add zeros. This is especially useful together with abs_value for gesting logaritmic.In general, the data set has to have following structure:
ATTR_DATE_TIME. Datetime index as the index column.ATTR_ID as an integer identifier of the observation. This is necessary for cross validation and models and pipelines assessment for better work than with data time format.import sys
import os
sys.path+=[os.path.join(os.getcwd(), ".."), os.path.join(os.getcwd(), "../..")] # one and two up
ToC
Necessary libraries for notebook functionality:
NOTE: This way, using the function, the button works only in active notebook. If the functionality needs to be preserved in html export, then the code has to be incluced directly into notebook.
LOGGER_CONFIG_NAME = "logger_file_limit_console"
ADDAPT_WIDTH = False
try:
from src.utils.notebook_support_functions import create_button, get_notebook_name
NOTEBOOK_NAME = get_notebook_name()
SUPPORT_FUNCTIONS_READ = True
except:
NOTEBOOK_NAME = "NO_NAME"
SUPPORT_FUNCTIONS_READ = False
from src.utils.logger import Logger
from src.utils.envs import Envs
from src.utils.config import Config
from pandas import options
from IPython.display import display, HTML
options.display.max_rows = 500
options.display.max_columns = 500
envs = Envs()
envs.set_logger(LOGGER_CONFIG_NAME)
Logger().start_timer(f"NOTEBOOK; Notebook name: {NOTEBOOK_NAME}")
if ADDAPT_WIDTH:
display(HTML("<style>.container { width:100% !important; }</style>")) # notebook width
2023-12-10 16:22:47,226 - file_limit_console - INFO - Logger was created on WS-3000 in branche 015_update_repository. 2023-12-10 16:22:47,228 - file_limit_console - INFO - Process: NOTEBOOK; Notebook name: anomaly_detection_sample_data_documentation.py; Timer started;
# create_button()
from datetime import datetime
from src.utils.date_time_functions import create_datetime_id
from src.data.anomaly_detection_sample_data import AnomalyDetectionSampleData, plot_data_frame_series
from src.data.attributes import A
from src.data.anomaly_detection_sample_data import ATTRS
# from src.global_constants import * # Remember to import only the constants in use
N_ROWS_TO_DISPLAY = 2
FIGURE_SIZE_SETTING = {"autosize": False, "width": 2200, "height": 750}
DATA_PROCESSING_CONFIG_NAME = "data_processing_basic"
# MANDATORY FOR CONFIG DEFINITION AND NOTEBOOK AND ITS OUTPUTS IDENTIFICATION #########################################
PYTHON_CONFIG_NAME = "python_local"
ID = create_datetime_id(now=datetime.now(), add_micro=False)
# (END) MANDATORY FOR CONFIG DEFINITION AND NOTEBOOK AND ITS OUTPUTS IDENTIFICATION ###################################
ABS_VALUES = True
ADD_ZEROS = False
envs.set_config(PYTHON_CONFIG_NAME)
data = AnomalyDetectionSampleData(abs_values=ABS_VALUES, add_zeros=ADD_ZEROS)
df = data.generate(n=100, seed_number=376)
df.head()
| DATE_TIME | ID | ATTR_1 | ATTR_2 | ATTR_3 | ATTR_4 | ATTR_5 | |
|---|---|---|---|---|---|---|---|
| 1980-01-01 | 1980-01-01 | 0 | 0.205113 | 1.614052 | 0.040801 | 3.950864 | 6.171155 |
| 1980-01-02 | 1980-01-02 | 1 | 0.413252 | 3.429908 | 0.435559 | 4.442794 | 3.240599 |
| 1980-01-03 | 1980-01-03 | 2 | 1.759583 | 0.999676 | 0.483382 | 4.732806 | 6.224277 |
| 1980-01-04 | 1980-01-04 | 3 | 0.582578 | 0.024122 | 0.971083 | 1.130060 | 4.636617 |
| 1980-01-05 | 1980-01-05 | 4 | 1.001934 | 1.790439 | 1.550204 | 5.074024 | 6.039363 |
plot_data_frame_series(df, ATTRS)
data = AnomalyDetectionSampleData(abs_values=True, add_zeros=True)
df = data.generate(n=100, seed_number=376)
print(df.head(N_ROWS_TO_DISPLAY))
plot_data_frame_series(df, ATTRS)
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-01 1980-01-01 0 0.000000 0.000000 0.000000 0.000000 0.000000
1980-01-02 1980-01-02 1 0.205113 0.864108 0.836013 2.121971 5.649898
data = AnomalyDetectionSampleData(abs_values=False, add_zeros=True)
df = data.generate(n=100, seed_number=376)
print(df.head(N_ROWS_TO_DISPLAY))
plot_data_frame_series(df, ATTRS)
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-01 1980-01-01 0 0.000000 0.000000 0.000000 0.000000 0.000000
1980-01-02 1980-01-02 1 0.205113 0.864108 -0.836013 -2.121971 5.649898
data = AnomalyDetectionSampleData(abs_values=True, add_zeros=False)
df = data.generate(n=100, seed_number=376)
print(df.head(N_ROWS_TO_DISPLAY))
plot_data_frame_series(df, ATTRS)
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-01 1980-01-01 0 0.205113 1.614052 0.040801 3.950864 6.171155
1980-01-02 1980-01-02 1 0.413252 3.429908 0.435559 4.442794 3.240599
data = AnomalyDetectionSampleData(abs_values=False, add_zeros=False)
df = data.generate(n=100, seed_number=376)
print(df.head(N_ROWS_TO_DISPLAY))
plot_data_frame_series(df, ATTRS)
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-01 1980-01-01 0 0.205113 -1.614052 0.040801 3.950864 6.171155
1980-01-02 1980-01-02 1 -0.413252 3.429908 0.435559 4.442794 3.240599
data = AnomalyDetectionSampleData(abs_values=True, add_zeros=True)
dfs, df_whole = data.create_list_of_data_frames(n=10, seed_number=398)
dfs
[ DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-01 1980-01-01 0 0.000000 0.000000 0.000000 0.000000 0.000000
1980-01-02 1980-01-02 1 0.084916 4.008845 1.132482 1.034435 4.474932,
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-03 1980-01-03 2 0.523208 1.507756 0.190795 3.508464 6.398938
1980-01-04 1980-01-04 3 0.650780 0.920233 1.382264 1.655938 6.258425,
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-05 1980-01-05 4 0.146936 1.626209 0.630052 1.154256 5.207971
1980-01-06 1980-01-06 5 0.120465 2.703472 0.154063 6.588302 6.035776
1980-01-07 1980-01-07 6 0.439127 1.651572 0.858141 0.047432 4.396393,
DATE_TIME ID ATTR_1 ATTR_2 ATTR_3 ATTR_4 ATTR_5
1980-01-08 1980-01-08 7 0.427995 1.959231 0.325965 4.220763 5.663366
1980-01-09 1980-01-09 8 2.469552 4.549737 0.648149 3.640220 3.965369
1980-01-10 1980-01-10 9 0.838591 1.207663 1.682721 5.264073 4.611373]
df_whole
| DATE_TIME | ID | ATTR_1 | ATTR_2 | ATTR_3 | ATTR_4 | ATTR_5 | |
|---|---|---|---|---|---|---|---|
| 1980-01-01 | 1980-01-01 | 0 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| 1980-01-02 | 1980-01-02 | 1 | 0.084916 | 4.008845 | 1.132482 | 1.034435 | 4.474932 |
| 1980-01-03 | 1980-01-03 | 2 | 0.523208 | 1.507756 | 0.190795 | 3.508464 | 6.398938 |
| 1980-01-04 | 1980-01-04 | 3 | 0.650780 | 0.920233 | 1.382264 | 1.655938 | 6.258425 |
| 1980-01-05 | 1980-01-05 | 4 | 0.146936 | 1.626209 | 0.630052 | 1.154256 | 5.207971 |
| 1980-01-06 | 1980-01-06 | 5 | 0.120465 | 2.703472 | 0.154063 | 6.588302 | 6.035776 |
| 1980-01-07 | 1980-01-07 | 6 | 0.439127 | 1.651572 | 0.858141 | 0.047432 | 4.396393 |
| 1980-01-08 | 1980-01-08 | 7 | 0.427995 | 1.959231 | 0.325965 | 4.220763 | 5.663366 |
| 1980-01-09 | 1980-01-09 | 8 | 2.469552 | 4.549737 | 0.648149 | 3.640220 | 3.965369 |
| 1980-01-10 | 1980-01-10 | 9 | 0.838591 | 1.207663 | 1.682721 | 5.264073 | 4.611373 |
Logger().end_timer()
2023-12-10 16:22:48,937 - file_limit_console - INFO - Process: NOTEBOOK; Notebook name: anomaly_detection_sample_data_documentation.py; Timer ended; Process Duration [s]: 1.71; Process Duration [m]: 0.03